{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "pycharm": {
     "name": "#%% md\n"
    }
   },
   "source": [
    "# Chapter 4"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true,
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "# LSTM 理论网址 https://blog.csdn.net/shijing_0214/article/details/52081301\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.nn.functional as F\n",
    "import torch.optim as optim"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['I', 'like', 'apple']"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "text/plain": [
       "['I', 'like', 'banana']"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "text/plain": [
       "[['I', 'like', 'apple'], ['I', 'like', 'banana']]"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "a='I like apple'.split()\n",
    "a\n",
    "b= 'I like banana'.split()\n",
    "b\n",
    "[a,b]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true,
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "# word → index 0123 → word_embedding table tensor vector"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true,
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "?nn.LSTM()\n",
    "'''\n",
    "inputs : x,(h_0,c_0) \n",
    "        x.shape (seq_len, batch, input_size)\n",
    "        h_0/c_0.shape (num_layers * num_directions, batch, hidden_size)\n",
    "outputs: output, (h_n, c_n)\n",
    "        output.shape (seq_len, batch, num_directions * hidden_size)\n",
    "        h_0/c_0.shape (num_layers * num_directions, batch, hidden_size)\n",
    "'''"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[tensor([[0.5448, 0.2335, 1.2606]]),\n",
       " tensor([[-0.5989,  0.7906, -0.9305]]),\n",
       " tensor([[-0.9157,  0.2502, -1.3181]]),\n",
       " tensor([[-0.0852, -0.8907,  0.0124]]),\n",
       " tensor([[ 1.4117, -0.2501, -0.3247]])]"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "lstm = nn.LSTM(3,3) # 输入维度既embedding_dim 输出维度 为 3既hidden_size\n",
    "inputs = [torch.randn(1,3) for _ in range(5)] # 序列长度为5\n",
    "inputs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(tensor([[[-0.4546,  0.5349, -0.2908]]]), tensor([[[1.7071, 1.2008, 1.7247]]]))"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 随机初始化隐藏状态\n",
    "hidden = (torch.randn(1,1,3),torch.randn(1,1,3))\n",
    "hidden"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[0.5448, 0.2335, 1.2606]])"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "inputs[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true,
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "# 另外我们可以对一整个序列进行训练.\n",
    "# LSTM第一个返回的第一个值是所有时刻的隐藏状态\n",
    "# 第二个返回值是最后一个时刻的隐藏状态\n",
    "#(所以\"out\"的最后一个和\"hidden\"是一样的)\n",
    "# 之所以这样设计:\n",
    "# 通过\"out\"你能取得任何一个时刻的隐藏状态，而\"hidden\"的值是用来进行序列的反向传播运算, 具体方式就是将它作为参数传入后面的 LSTM 网络.\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": true,
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "*******out*******\n",
      "tensor([[[0.1759, 0.5024, 0.5494]]], grad_fn=<StackBackward>)\n",
      "*******hidden*******\n",
      "(tensor([[[0.1759, 0.5024, 0.5494]]], grad_fn=<StackBackward>), tensor([[[0.7353, 0.9056, 0.8950]]], grad_fn=<StackBackward>))\n",
      "*******out*******\n",
      "tensor([[[-0.0567,  0.0749,  0.3343]]], grad_fn=<StackBackward>)\n",
      "*******hidden*******\n",
      "(tensor([[[-0.0567,  0.0749,  0.3343]]], grad_fn=<StackBackward>), tensor([[[-0.2780,  0.1960,  0.8750]]], grad_fn=<StackBackward>))\n",
      "*******out*******\n",
      "tensor([[[-0.1711,  0.0063,  0.3382]]], grad_fn=<StackBackward>)\n",
      "*******hidden*******\n",
      "(tensor([[[-0.1711,  0.0063,  0.3382]]], grad_fn=<StackBackward>), tensor([[[-0.8124,  0.0191,  0.8371]]], grad_fn=<StackBackward>))\n",
      "*******out*******\n",
      "tensor([[[-0.1853,  0.0966,  0.3972]]], grad_fn=<StackBackward>)\n",
      "*******hidden*******\n",
      "(tensor([[[-0.1853,  0.0966,  0.3972]]], grad_fn=<StackBackward>), tensor([[[-0.7110,  0.1801,  0.6110]]], grad_fn=<StackBackward>))\n",
      "*******out*******\n",
      "tensor([[[-0.2245,  0.1125,  0.2626]]], grad_fn=<StackBackward>)\n",
      "*******hidden*******\n",
      "(tensor([[[-0.2245,  0.1125,  0.2626]]], grad_fn=<StackBackward>), tensor([[[-0.5055,  0.1802,  0.4961]]], grad_fn=<StackBackward>))\n"
     ]
    }
   ],
   "source": [
    "for i in inputs:\n",
    "    out,hidden = lstm(i.view(1,1,-1),hidden)\n",
    "    print('*******out*******')\n",
    "    print(out)\n",
    "    print('*******hidden*******')\n",
    "    print(hidden)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor([[[-0.0637,  0.4562, -0.5386]],\n",
      "\n",
      "        [[-0.1926,  0.1307, -0.1278]],\n",
      "\n",
      "        [[-0.2296,  0.0446,  0.0724]],\n",
      "\n",
      "        [[-0.2140,  0.1362,  0.1925]],\n",
      "\n",
      "        [[-0.2380,  0.1501,  0.1498]]], grad_fn=<StackBackward>)\n",
      "(tensor([[[-0.2380,  0.1501,  0.1498]]], grad_fn=<StackBackward>), tensor([[[-0.5053,  0.2408,  0.2659]]], grad_fn=<StackBackward>))\n"
     ]
    }
   ],
   "source": [
    "inputs= torch.cat(inputs).view(len(inputs),1,-1)\n",
    "hidden = (torch.randn(1,1,3),torch.randn(1,1,3))\n",
    "out,hidden = lstm(inputs,hidden)\n",
    "print(out)\n",
    "print(hidden)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "collapsed": true,
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "# 词性标注 apple n eat v easily adv.\n",
    "training_data = [\n",
    "    (\"The dog ate the apple\".split(), [\"DET\", \"NN\", \"V\", \"DET\", \"NN\"]),\n",
    "    (\"Everybody read that book\".split(), [\"NN\", \"V\", \"DET\", \"NN\"])\n",
    "]\n",
    "word_to_ix = {}\n",
    "for sent,tags in training_data:\n",
    "    for word in sent:\n",
    "        if word not in word_to_ix:\n",
    "            word_to_ix[word]=len(word_to_ix)\n",
    "def prepare_sequence(seq,to_ix):\n",
    "    idxs = [to_ix[w] for w in seq]\n",
    "    return torch.tensor(idxs,dtype=torch.long)\n",
    "tag_to_ix ={\"DET\":0,\"NN\":1,\"V\":2}\n",
    "EMBEDDING_DIM = 6\n",
    "HIDDEN_SIZE = 6"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "collapsed": true,
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "class LSTMTagger(nn.Module):\n",
    "    def __init__(self,vocab_size,embedding_dim,hidden_size,tagset_size):\n",
    "        super(LSTMTagger,self).__init__()\n",
    "        self.word_embeddings = nn.Embedding(vocab_size,embedding_dim)\n",
    "        self.lstm = nn.LSTM(embedding_dim,hidden_size)\n",
    "        self.hidden2tag = nn.Linear(hidden_size,tagset_size)\n",
    "        self.hidden_size = hidden_size\n",
    "        self.hidden = self.init_hidden()\n",
    "    def init_hidden(self):\n",
    "        return (torch.zeros(1,1,self.hidden_size),torch.zeros(1,1,self.hidden_size))\n",
    "    def forward(self,sentence):\n",
    "        embeds = self.word_embeddings(sentence).view(len(sentence),1,-1)\n",
    "        lstm_out,self.hidden = self.lstm(embeds,self.hidden)\n",
    "        tag_space = self.hidden2tag(lstm_out.view(len(sentence),-1)) # 5 1 3 → 5 3\n",
    "        tag_scores = F.log_softmax(tag_space,dim = 1)\n",
    "        return tag_scores"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "collapsed": true,
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "model = LSTMTagger(len(word_to_ix),EMBEDDING_DIM,HIDDEN_SIZE,len(tag_to_ix))\n",
    "loss_function = nn.NLLLoss()\n",
    "optimizer = optim.SGD(model.parameters(),lr=0.1)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['The', 'dog', 'ate', 'the', 'apple']\n",
      "{'DET': 0, 'NN': 1, 'V': 2}\n",
      "tensor([[-1.0657, -1.2878, -0.9686],\n",
      "        [-1.0375, -1.3901, -0.9248],\n",
      "        [-0.9728, -1.4088, -0.9741],\n",
      "        [-1.0091, -1.3434, -0.9822],\n",
      "        [-0.9720, -1.2979, -1.0540]])\n"
     ]
    }
   ],
   "source": [
    "with torch.no_grad():\n",
    "    inputs = prepare_sequence(training_data[0][0],word_to_ix)\n",
    "    tag_scores = model(inputs)\n",
    "    print(training_data[0][0])\n",
    "    print(tag_to_ix)\n",
    "    print(tag_scores)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {
    "scrolled": true,
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1.9666758179664612\n",
      "1.9610134363174438\n",
      "1.9551985263824463\n",
      "1.9492259621620178\n",
      "1.9430906772613525\n",
      "1.9367880821228027\n",
      "1.9303141236305237\n",
      "1.9236646890640259\n",
      "1.9168368577957153\n",
      "1.9098273515701294\n",
      "1.9026339054107666\n",
      "1.895254373550415\n",
      "1.8876876831054688\n",
      "1.8799326419830322\n",
      "1.8719887733459473\n",
      "1.863856554031372\n",
      "1.8555367588996887\n",
      "1.847029685974121\n",
      "1.8383375406265259\n",
      "1.8294617533683777\n",
      "1.820404291152954\n",
      "1.8111674785614014\n",
      "1.8017538189888\n",
      "1.7921655774116516\n",
      "1.7824051976203918\n",
      "1.7724751830101013\n",
      "1.7623786330223083\n",
      "1.7521171569824219\n",
      "1.741694152355194\n",
      "1.7311118841171265\n",
      "1.7203731536865234\n",
      "1.7094807624816895\n",
      "1.6984370946884155\n",
      "1.6872453093528748\n",
      "1.6759083271026611\n",
      "1.664429247379303\n",
      "1.6528112888336182\n",
      "1.6410577893257141\n",
      "1.6291720867156982\n",
      "1.6171578168869019\n",
      "1.6050189137458801\n",
      "1.5927591919898987\n",
      "1.5803824663162231\n",
      "1.5678929090499878\n",
      "1.5552950501441956\n",
      "1.5425926446914673\n",
      "1.5297905802726746\n",
      "1.516893208026886\n",
      "1.503905177116394\n",
      "1.4908311367034912\n",
      "1.47767573595047\n",
      "1.4644436240196228\n",
      "1.4511393904685974\n",
      "1.4377679228782654\n",
      "1.4243335723876953\n",
      "1.4108409881591797\n",
      "1.3972944617271423\n",
      "1.383698046207428\n",
      "1.3700560331344604\n",
      "1.3563718795776367\n",
      "1.3426489233970642\n",
      "1.3288904428482056\n",
      "1.3150994181632996\n",
      "1.3012778162956238\n",
      "1.2874279022216797\n",
      "1.273551106452942\n",
      "1.259648621082306\n",
      "1.24572092294693\n",
      "1.2317686676979065\n",
      "1.217791199684143\n",
      "1.2037885785102844\n",
      "1.1897592544555664\n",
      "1.1757025122642517\n",
      "1.1616167426109314\n",
      "1.1475005149841309\n",
      "1.1333519220352173\n",
      "1.1191692352294922\n",
      "1.1049508154392242\n",
      "1.0906952023506165\n",
      "1.0764010846614838\n",
      "1.0620679259300232\n",
      "1.0476950109004974\n",
      "1.0332832038402557\n",
      "1.0188333690166473\n",
      "1.0043476819992065\n",
      "0.9898292422294617\n",
      "0.9752823710441589\n",
      "0.9607124328613281\n",
      "0.946126252412796\n",
      "0.9315318465232849\n",
      "0.916938453912735\n",
      "0.9023566246032715\n",
      "0.8877980709075928\n",
      "0.8732752501964569\n",
      "0.8588016629219055\n",
      "0.8443909883499146\n",
      "0.8300575017929077\n",
      "0.8158152103424072\n",
      "0.8016782104969025\n",
      "0.7876600921154022\n"
     ]
    }
   ],
   "source": [
    "for epoch in range(100):\n",
    "    total_loss = 0\n",
    "    for sentence,tags in training_data:\n",
    "        model.zero_grad()\n",
    "        sentence_in = prepare_sequence(sentence,word_to_ix)\n",
    "        targets = prepare_sequence(tags,tag_to_ix)\n",
    "        model.hidden = model.init_hidden()\n",
    "        tag_scores = model(sentence_in)\n",
    "        loss = loss_function(tag_scores,targets)\n",
    "        loss.backward()\n",
    "        optimizer.step()\n",
    "        total_loss += loss.item()\n",
    "    print(total_loss)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['The', 'dog', 'ate', 'the', 'apple']\n",
      "{'DET': 0, 'NN': 1, 'V': 2}\n",
      "tensor([[-1.4350, -0.3643, -2.6995],\n",
      "        [-2.3176, -0.1589, -3.0280],\n",
      "        [-1.7255, -2.4228, -0.3103],\n",
      "        [-0.5242, -1.3069, -1.9852],\n",
      "        [-1.8240, -0.1860, -4.7860]])\n"
     ]
    }
   ],
   "source": [
    "with torch.no_grad():\n",
    "    inputs = prepare_sequence(training_data[0][0],word_to_ix)\n",
    "    tag_scores = model(inputs)\n",
    "    print(training_data[0][0])\n",
    "    print(tag_to_ix)\n",
    "    print(tag_scores)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true,
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}